{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 实验环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from imblearn.over_sampling import SMOTE#采样\n",
    "\n",
    "from sklearn.svm import SVC #支持向量机\n",
    "from sklearn.linear_model import LogisticRegression #线性回归\n",
    "from sklearn.neural_network import MLPClassifier #MLP神经网络\n",
    "from sklearn.ensemble import RandomForestClassifier #随机森林\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold #划分数据集，分层采样交叉切分\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler#标准化数据\n",
    "from sklearn.metrics import precision_score, recall_score, roc_auc_score, average_precision_score, f1_score#评估"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>10</td>\n",
       "      <td>101</td>\n",
       "      <td>76</td>\n",
       "      <td>48</td>\n",
       "      <td>180</td>\n",
       "      <td>32.9</td>\n",
       "      <td>0.171</td>\n",
       "      <td>63</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>2</td>\n",
       "      <td>122</td>\n",
       "      <td>70</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>36.8</td>\n",
       "      <td>0.340</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>5</td>\n",
       "      <td>121</td>\n",
       "      <td>72</td>\n",
       "      <td>23</td>\n",
       "      <td>112</td>\n",
       "      <td>26.2</td>\n",
       "      <td>0.245</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>1</td>\n",
       "      <td>126</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30.1</td>\n",
       "      <td>0.349</td>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>1</td>\n",
       "      <td>93</td>\n",
       "      <td>70</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>30.4</td>\n",
       "      <td>0.315</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>768 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0              6      148             72             35        0  33.6   \n",
       "1              1       85             66             29        0  26.6   \n",
       "2              8      183             64              0        0  23.3   \n",
       "3              1       89             66             23       94  28.1   \n",
       "4              0      137             40             35      168  43.1   \n",
       "..           ...      ...            ...            ...      ...   ...   \n",
       "763           10      101             76             48      180  32.9   \n",
       "764            2      122             70             27        0  36.8   \n",
       "765            5      121             72             23      112  26.2   \n",
       "766            1      126             60              0        0  30.1   \n",
       "767            1       93             70             31        0  30.4   \n",
       "\n",
       "     DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                       0.627   50        1  \n",
       "1                       0.351   31        0  \n",
       "2                       0.672   32        1  \n",
       "3                       0.167   21        0  \n",
       "4                       2.288   33        1  \n",
       "..                        ...  ...      ...  \n",
       "763                     0.171   63        0  \n",
       "764                     0.340   27        0  \n",
       "765                     0.245   30        0  \n",
       "766                     0.349   47        1  \n",
       "767                     0.315   23        0  \n",
       "\n",
       "[768 rows x 9 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def LoadData():\n",
    "    rawDataset = pd.read_csv('diabetes.csv')\n",
    "    return rawDataset\n",
    "LoadData()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ 数据中不存在缺失值\n",
    "+ 数据的各个属性值均大于零，且均为连续值\n",
    "+ 数据的各个属性分布较为不均衡，体现为不同属性的均值和标准差相差较大\n",
    "+ 通过观察数据的同一个属性下的均值和方差，我们任务部分数据存在异常值\n",
    "+ 样本正负比例不均衡，多数样本为负样本"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  异常点去除"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![turky箱图](https://gitee.com/ZY13021877/gitee_picture/raw/master/Turky%E7%AE%B1%E5%9B%BE.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IQR准则\n",
    "#四分位间距(Q1,Q3)涵盖了数据分布最中间的50%的数据，具有稳健性；\n",
    "#数据落在 (Q1-1.5*IQR，Q3+1.5∗IQR) 范围内，则认为是正常值;\n",
    "#在此范围之外的即为异常值;\n",
    "#首先求出了所有异常点的编号位置，而后对异常值进行统计，对超出阈值次数的异常值予以剔除\n",
    "from collections import Counter  \n",
    "def Preprocess(rawDataset, threshold):#rawDataset的数据形式，字典key为特征名，value为数值\n",
    "    outlierIndices = []#建立一个空列表\n",
    "    features = list(rawDataset)[:-1]#切片[ )，9个列名，取前8个列名为特征名\n",
    "    for feature in features:#for循环依次取特征\n",
    "        Q1 = np.percentile(rawDataset[feature], 25)\n",
    "        Q3 = np.percentile(rawDataset[feature],75)\n",
    "        IQR = Q3 - Q1#数据分布最中间的50%的数据，稳健\n",
    "        outlierStep = 1.5 * IQR\n",
    "        outlierIndices.extend(rawDataset[(rawDataset[feature] < Q1 - outlierStep) \n",
    "        | (rawDataset[feature] > Q3 + outlierStep )].index)#列表extend方法，把rawDataset中异常值的序号添加到outlierIndices列表\n",
    "    outlierIndices = Counter(outlierIndices)#Counter求数组中每个数字出现了几次\n",
    "    outlierIndices = list(k for k, v in outlierIndices.items() if v > threshold )#求出超出阈值次数的异常值的序号\n",
    "    rawDataset.drop(outlierIndices, axis = 0).reset_index(drop=True)#对超出阈值次数的异常值予以剔除，0表示沿着行向下\n",
    "    dataset = rawDataset.values[:,:-1]#所有行，除最后一列\n",
    "    label = rawDataset.values[:,-1]#所有行，最后一列\n",
    "    return dataset, label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#使用sklearn库构建包括：逻辑回归、支持向量机、随机森林和多层感知机在内的多种模型；\n",
    "#为了方便调整参数，将随机种子random_state固定为0。\n",
    "def BuidModel(modelKey):\n",
    "    if modelKey == 'svm':\n",
    "        return SVC(probability=True, random_state=0)\n",
    "    elif modelKey == 'rf':\n",
    "        return RandomForestClassifier(n_estimators=512, random_state=0)\n",
    "    elif modelKey == 'mlp':\n",
    "        return MLPClassifier(max_iter=10000, random_state=0)\n",
    "    elif modelKey == 'logistic':\n",
    "        return LogisticRegression(random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型训练&评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#评估模型，设计了多个模型\n",
    "#模型和数据采用 交叉验证框架方式  对模型进行评估\n",
    "'''\n",
    "采用精准率(precision)，召回率(recall)，\n",
    "受试者操作特征曲线下的面积(the area under the receiver operating characteristic curve)，\n",
    "平均精度(average precision)（也就是PR曲线下方的面积AUPR）\n",
    "进行模型评估\n",
    "'''\n",
    "def TrainTestModel(dataset, label, processKey, modelKey, K):# dataset数据 label标签 processKey标准化策略  modelKey模型 K表示进行几次评估\n",
    "    #训练集评估变量\n",
    "    trainingPrecision = np.zeros(K)#训练集准确率#np.zeros(K):array([0,0,0,0,0])一行K列的用0填充的数组\n",
    "    trainingRecall = np.zeros(K)#召回率\n",
    "    traingAuc = np.zeros(K)\n",
    "    traingAupr = np.zeros(K)\n",
    "    traingF1 = np.zeros(K)\n",
    "    #测试集评估变量\n",
    "    validationPrecision = np.zeros(K)#测试集\n",
    "    validationRecall = np.zeros(K)\n",
    "    validationAuc = np.zeros(K)\n",
    "    validationAupr = np.zeros(K)\n",
    "    validationF1 = np.zeros(K)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    #分层采样交叉切分\n",
    "    #确保训练集，测试集中各类别样本的比例与原始数据集中相同\n",
    "    kFold = StratifiedKFold(n_splits=K)\n",
    "    for (k, (trainingIndex, validationIndex)) in enumerate(kFold.split(dataset, label)):#enumerate相当于加序号\n",
    "    #for train, test in sfolder.split(X,y):\n",
    "        trainingData, validationData = dataset[trainingIndex], dataset[validationIndex]\n",
    "        trainingLabel, validationLabel = label[trainingIndex], label[validationIndex]\n",
    "\n",
    "        #数据标准化\n",
    "        #正态尺度标准化和最小最大尺度标准化来对数据进行处理 \n",
    "        if processKey == 'standard':\n",
    "            scaler = StandardScaler()\n",
    "        elif processKey == 'minMax':\n",
    "            scaler = MinMaxScaler()\n",
    "        trainingData = scaler.fit_transform(trainingData)\n",
    "        validationData = scaler.transform(validationData)\n",
    "        \n",
    "        #采样训练集  \n",
    "        #在这之前，划分了训练集和测试集，仅仅对训练集的数据进行上采样\n",
    "        #针对于数据集正负样本不均衡的问题，采用SMOTE采样方法\n",
    "        #消除由于采样和数据划分带来的数据依赖性，使实验结果更加准确\n",
    "        smo = SMOTE(random_state=0)\n",
    "        trainingData, trainingLabel = smo.fit_resample(trainingData, trainingLabel)\n",
    "\n",
    "        #建立模型，训练模型\n",
    "        model = BuidModel(modelKey)\n",
    "        model.fit(trainingData, trainingLabel)\n",
    "\n",
    "        \n",
    "        traingingProbablity = model.predict_proba(trainingData)[:, 1]\n",
    "        #predict_proba返回的是一个n行k列的数组，\n",
    "        #只取所有行第一列，即为结果为0的概率\n",
    "        traingPredictLabel = model.predict(trainingData)\n",
    "        #预测出为0还是1，返回结果\n",
    "        trainingPrecision[k] = precision_score(trainingLabel, traingPredictLabel)\n",
    "        #Precision(精确率)\n",
    "        trainingRecall[k] = recall_score(trainingLabel, traingPredictLabel)\n",
    "        #召回率\n",
    "        traingAuc[k] = roc_auc_score(trainingLabel, traingingProbablity)\n",
    "        #ROC曲线面积\n",
    "        traingAupr[k] = average_precision_score(trainingLabel, traingingProbablity)\n",
    "        #平均精度(average precision)（也就是PR曲线下方的面积AUPR）\n",
    "        traingF1[k] = f1_score(trainingLabel, traingPredictLabel)\n",
    "        #F1分数可以看作是模型精确确率和召回率的一种加权平均，它的最大值是1，最小值是0.\n",
    "\n",
    "        print('*****************k={0}*****************'.format(k))\n",
    "        print('Training Precision:', trainingPrecision[k])\n",
    "        print('Training Recall:', trainingRecall[k])\n",
    "        print('Training auc:', traingAuc[k])\n",
    "        print('Training aupr:', traingAupr[k])\n",
    "        print('Training f1:', traingF1[k])\n",
    "        \n",
    "        validationProbablity = model.predict_proba(validationData)[:, 1]\n",
    "        validationPredictLabel = model.predict(validationData)\n",
    "\n",
    "        validationPrecision[k] = precision_score(validationLabel, validationPredictLabel)\n",
    "        validationRecall[k] = recall_score(validationLabel, validationPredictLabel)\n",
    "        validationAuc[k] = roc_auc_score(validationLabel, validationProbablity)\n",
    "        validationAupr[k] = average_precision_score(validationLabel, validationProbablity)\n",
    "        validationF1[k] = f1_score(validationLabel, validationPredictLabel)\n",
    "\n",
    "        print('Validation Precision:', validationPrecision[k])\n",
    "        print('Validation Recall:', validationRecall[k])\n",
    "        print('Validation auc:', validationAuc[k])\n",
    "        print('Validation aupr:', validationAupr[k])\n",
    "        print('Validation f1:', validationF1[k])\n",
    "    \n",
    "\n",
    "    print('**********************************')\n",
    "    print('**********************************')\n",
    "    print('**********************************')\n",
    "    print('Mean Training Precision:', trainingPrecision.mean())\n",
    "    print('Mean Training Recall:', trainingRecall.mean())\n",
    "    print('Mean Training auc:', traingAuc.mean())\n",
    "    print('Mean Training aupr:', traingAupr.mean())\n",
    "    print('Mean Training f1:', traingF1.mean())\n",
    "\n",
    "    print('Mean Validation Precision:', validationPrecision.mean())\n",
    "    print('Mean Validation Recall:', validationRecall.mean())\n",
    "    print('Mean Validation auc:', validationAuc.mean())\n",
    "    print('Mean Validation aupr:', validationAupr.mean())\n",
    "    print('Mean Validation f1:', validationF1.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*****************k=0*****************\n",
      "Training Precision: 0.7788018433179723\n",
      "Training Recall: 0.7511111111111111\n",
      "Training auc: 0.8526370370370371\n",
      "Training aupr: 0.8297431003127342\n",
      "Training f1: 0.7647058823529411\n",
      "Validation Precision: 0.5714285714285714\n",
      "Validation Recall: 0.7407407407407407\n",
      "Validation auc: 0.8037037037037037\n",
      "Validation aupr: 0.7256850228837795\n",
      "Validation f1: 0.6451612903225806\n",
      "*****************k=1*****************\n",
      "Training Precision: 0.7795454545454545\n",
      "Training Recall: 0.7622222222222222\n",
      "Training auc: 0.8499703703703704\n",
      "Training aupr: 0.8290171420566803\n",
      "Training f1: 0.7707865168539326\n",
      "Validation Precision: 0.6923076923076923\n",
      "Validation Recall: 0.6666666666666666\n",
      "Validation auc: 0.8148148148148148\n",
      "Validation aupr: 0.7587969155910737\n",
      "Validation f1: 0.6792452830188679\n",
      "*****************k=2*****************\n",
      "Training Precision: 0.7741935483870968\n",
      "Training Recall: 0.7466666666666667\n",
      "Training auc: 0.8468345679012346\n",
      "Training aupr: 0.8293515281952981\n",
      "Training f1: 0.7601809954751131\n",
      "Validation Precision: 0.6060606060606061\n",
      "Validation Recall: 0.7407407407407407\n",
      "Validation auc: 0.8466666666666667\n",
      "Validation aupr: 0.725264491270496\n",
      "Validation f1: 0.6666666666666666\n",
      "*****************k=3*****************\n",
      "Training Precision: 0.7780373831775701\n",
      "Training Recall: 0.74\n",
      "Training auc: 0.853130864197531\n",
      "Training aupr: 0.8336999242021322\n",
      "Training f1: 0.7585421412300682\n",
      "Validation Precision: 0.5277777777777778\n",
      "Validation Recall: 0.7037037037037037\n",
      "Validation auc: 0.7540740740740741\n",
      "Validation aupr: 0.6274968659420228\n",
      "Validation f1: 0.6031746031746033\n",
      "*****************k=4*****************\n",
      "Training Precision: 0.7757009345794392\n",
      "Training Recall: 0.7377777777777778\n",
      "Training auc: 0.8498567901234567\n",
      "Training aupr: 0.8269032603251051\n",
      "Training f1: 0.7562642369020501\n",
      "Validation Precision: 0.5714285714285714\n",
      "Validation Recall: 0.5925925925925926\n",
      "Validation auc: 0.7962962962962963\n",
      "Validation aupr: 0.7197997266426064\n",
      "Validation f1: 0.5818181818181818\n",
      "*****************k=5*****************\n",
      "Training Precision: 0.7638888888888888\n",
      "Training Recall: 0.7333333333333333\n",
      "Training auc: 0.8440345679012347\n",
      "Training aupr: 0.8211510489938374\n",
      "Training f1: 0.7482993197278911\n",
      "Validation Precision: 0.6451612903225806\n",
      "Validation Recall: 0.7407407407407407\n",
      "Validation auc: 0.8548148148148148\n",
      "Validation aupr: 0.7816674617757291\n",
      "Validation f1: 0.689655172413793\n",
      "*****************k=6*****************\n",
      "Training Precision: 0.7517564402810304\n",
      "Training Recall: 0.7133333333333334\n",
      "Training auc: 0.8457728395061729\n",
      "Training aupr: 0.8225843455688012\n",
      "Training f1: 0.7320410490307868\n",
      "Validation Precision: 0.6896551724137931\n",
      "Validation Recall: 0.7407407407407407\n",
      "Validation auc: 0.8088888888888888\n",
      "Validation aupr: 0.6660335118185174\n",
      "Validation f1: 0.7142857142857143\n",
      "*****************k=7*****************\n",
      "Training Precision: 0.7464788732394366\n",
      "Training Recall: 0.7066666666666667\n",
      "Training auc: 0.8370123456790124\n",
      "Training aupr: 0.813932796528752\n",
      "Training f1: 0.7260273972602739\n",
      "Validation Precision: 0.7419354838709677\n",
      "Validation Recall: 0.8518518518518519\n",
      "Validation auc: 0.9059259259259259\n",
      "Validation aupr: 0.8419001878854815\n",
      "Validation f1: 0.7931034482758621\n",
      "*****************k=8*****************\n",
      "Training Precision: 0.7523364485981309\n",
      "Training Recall: 0.7155555555555555\n",
      "Training auc: 0.8405728395061729\n",
      "Training aupr: 0.8209595861106735\n",
      "Training f1: 0.733485193621868\n",
      "Validation Precision: 0.6666666666666666\n",
      "Validation Recall: 0.7692307692307693\n",
      "Validation auc: 0.8546153846153846\n",
      "Validation aupr: 0.6580292698414197\n",
      "Validation f1: 0.7142857142857142\n",
      "*****************k=9*****************\n",
      "Training Precision: 0.7534562211981567\n",
      "Training Recall: 0.7266666666666667\n",
      "Training auc: 0.8400740740740741\n",
      "Training aupr: 0.8161989532377201\n",
      "Training f1: 0.7398190045248869\n",
      "Validation Precision: 0.6\n",
      "Validation Recall: 0.6923076923076923\n",
      "Validation auc: 0.8346153846153846\n",
      "Validation aupr: 0.731246489597415\n",
      "Validation f1: 0.6428571428571429\n",
      "**********************************\n",
      "**********************************\n",
      "**********************************\n",
      "Mean Training Precision: 0.7654196036213177\n",
      "Mean Training Recall: 0.7333333333333333\n",
      "Mean Training auc: 0.8459896296296296\n",
      "Mean Training aupr: 0.8243541685531734\n",
      "Mean Training f1: 0.7490151736979812\n",
      "Mean Validation Precision: 0.6312421832277227\n",
      "Mean Validation Recall: 0.7239316239316239\n",
      "Mean Validation auc: 0.8274415954415956\n",
      "Mean Validation aupr: 0.7235919943248541\n",
      "Mean Validation f1: 0.6730253217119128\n"
     ]
    }
   ],
   "source": [
    "rawDataset = LoadData()\n",
    "dataset, label = Preprocess(rawDataset, 1)\n",
    "#数据进行预处理选取离群点的阈值为1，任何存在两个异常属性及以上的记录将会被移除\n",
    "TrainTestModel(dataset, label, 'minMax', 'logistic', K=10)\n",
    "#最小最大尺度对数据进行标准化\n",
    "#模型-modelKey\n",
    "#10折交叉验证框架对模型进行评估"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "运行10次10交叉实验（10-CV）以避免偏差，系统性的评估不同的机器学习模型  \n",
    "从上述结果中我们可以发现：  \n",
    "所有的模型都在我们的数据集上均产生了大体令人满意的结果   \n",
    "不同的模型出现了不同程度的过拟合，表现为在训练集上的性能高于测试集上的性能。  \n",
    "其中逻辑回归过拟合成都较小，随机森林过拟合程度较大。  \n",
    "这是由于相比于随机森林，逻辑回归的模型参数量较小，拟合能力有限，不容易发生过拟合。  \n",
    "逻辑回归以较少的参数量，较短的训练时间产生了较为满意的结果。这一点体现了奥卡姆剃刀定律。\n",
    "在测试数据上，四种模型都实现了相近的效果，整体的性能不会因为采用了不同的模型存在较大的变化；\n",
    "要想获得较好的性能需要选择一个模型然后不断地尝试和优化。这一点体现了没有免费午餐定理（NFL）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
